package crawler;

import java.io.IOException;
import java.util.ArrayList;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
import tools.FunctionUtils;
import tools.UrlQueue;
import tools.VisitedUrlQueue;
import crawler.ctdata.CtData;
import crawler.ctkeyword.CtKeyword;
import crawler.ctrules.CtRules;
import crawler.ctweb.CtWeb;

public class Crawler {
	private CtWeb web;
	private ArrayList<CtRules> rulesArr;
	private ArrayList<CtData> dataArr;
	private Boolean isHasDataArr = false;
	public Crawler(CtWeb web , ArrayList<CtRules> rulesArr){
		this.web = web;
		this.rulesArr = rulesArr;
	}
	
	/**
	 * 读取网页源码
	 * @param url
	 * @return
	 */
	private String readWebHTMLString(String url){
		HttpClient client = new DefaultHttpClient();
		HttpGet gethttp = new HttpGet(url);
		String content = null;
		HttpResponse response;
		try{
			response = client.execute(gethttp);
			HttpEntity entity = response.getEntity();
			if (entity != null){
				content = EntityUtils.toString(entity); 
				if (FunctionUtils.isCreateFile(url)  && FunctionUtils.isHasGoalContent(content) !=  -1) { 
					FunctionUtils.createFile(FunctionUtils.getGoalContent(content), url);
				}
			}
		}catch(ClientProtocolException  e){
			e.printStackTrace();
		}catch(IOException  e){
			e.printStackTrace();
		}finally{
			client.getConnectionManager().shutdown();
		}
		return content;
	}
	
	/**
	 * 截取网页源码数据收集区域
	 * @param content
	 * @param interBegin
	 * @param interEnd
	 * @return
	 */
	private String interceptCollectArea(String content,String interBegin , String interEnd){
		String subContent = content.substring(content.indexOf(interBegin), content.indexOf(interEnd));
		return subContent;
	}
	
	/**
	 * 截取数据
	 * @param content
	 * @param name
	 * @param interBegin
	 * @param interEnd
	 * @return
	 */
	private CtData interceptData(String content,String name,String interBegin , String interEnd){
		CtData ctdata = new CtData();
		ctdata.setDtName(name);
		ctdata.setDtContent(content.substring(content.indexOf(interBegin), content.indexOf(interEnd)));
		return ctdata;
	}
	
	/**
	 * 截取数据
	 * @param content
	 * @param rules
	 * @return
	 */
	private CtData interceptData(String content , CtRules rules){
		return interceptData(content,rules.getCrName() , rules.getCrCtareaBegin() , rules.getCrCtareaEnd());
	}
	
	/**
	 * 批量截取数据
	 * @param content
	 * @param rulesArr
	 * @return
	 */
	private ArrayList<CtData> interceptBatchData(String content , ArrayList<CtRules> rulesArr){
		ArrayList<CtData> list = new ArrayList<CtData>();
		for(int i = 0 , j = rulesArr.size() ; i < j ; i++){
			CtRules rules = rulesArr.get(i);
			CtData data = interceptData(content,rules);
			list.add(data);
		}
		return list;
	}
	
	/**
	 * 截取网页URL
	 * @param content
	 * @return
	 */
	private ArrayList<String> interceptUrl(String content){
		ArrayList<String> urlArr = new ArrayList<String>();
		String[] contents = content.split("<a href=\"");
		for(int i = 1 ; i < content.length() ; i++){
			int endHref = contents[i].indexOf("\"");
			String aHref = FunctionUtils.getHrefOfInOut(contents[i].substring(0, endHref));
			if (aHref != null){
				String href = FunctionUtils.getHrefOfInOut(aHref);
				if (!urlArr.contains(href)){
					urlArr.add(href);
				}
			}
		}
		return urlArr;
	}
	
	/**
	 * 采集结束方法
	 * @param allData
	 */
	private void finishOp(ArrayList<CtData> allData){
		this.dataArr = allData;
		isHasDataArr = true;
	}
	
	/**
	 * 获取采集关键字
	 * @return
	 */
	public CtKeyword getCtKeyWord(){
		CtKeyword ctkeyword = new CtKeyword();
		if(isHasDataArr){
			for(int i = 0 ; i < dataArr.size() ; i++){
				for(int j = 0 ; j < rulesArr.size() ; j++){
					if(dataArr.get(i).getDtName().equals(rulesArr.get(j).getCrName())){
						ctkeyword.setKwContent(ctkeyword.getKwContent() + dataArr.get(i).getDtContent() + ",");
					}
				}
			}
		}
		return ctkeyword;
	}
	
	/**
	 * 截取数据Main方法
	 * @return
	 */
	public ArrayList<CtData> interceptDataMain(){
		String optype = web.getCwCttype();
		String paginNation = web.getCwPagination();
		ArrayList<String> webUrlArr = new ArrayList<String>();
		
		//01 分页采集 02 本页采集
		if("01".equals(paginNation)){
			String pagetype = web.getCwPagetype();
			String initpage = web.getCwInitpage();
			String finishpage = web.getCwFinishpage();
			String isZeros = web.getCwIszeros();
			String pageIncremental = web.getCwPageincremental();
			String weburl = web.getCwWeburl();
			
			if("01".equals(pagetype)){ //等差
				int init = Integer.parseInt(initpage);
				int finish = Integer.parseInt(finishpage);
				int incremental = Integer.parseInt(pageIncremental);
				for(int i = init , j = finish ; i <= j ; i+= incremental){
					webUrlArr.add(weburl.replace("(*)", i+""));
				}
			}else if("02".equals(pagetype)){//等比
				
			}else if("03".equals(pagetype)){//字母
				
			}
		}else{
			webUrlArr.add(web.getCwWeburl());
		}
		
		ArrayList<CtData> allData = new ArrayList<CtData>();
		for(int i = 0 ; i < webUrlArr.size() ; i++ ){
			String url = webUrlArr.get(i);
			String content = readWebHTMLString(url);
			// 01 ：采集URL 02 ：采集数据
			if("01".equals(optype)){
				ArrayList<String> urlArr = interceptUrl(content);
				for(int j = 0 ; j < urlArr.size() ; j++){
					String url2 = urlArr.get(j);
					content = readWebHTMLString(url2);
					ArrayList<CtData> data = interceptBatchData(content,rulesArr);
					allData.addAll(data);
				}
			}else{
				allData.addAll(interceptBatchData(content,rulesArr));
			}
		}
		
		finishOp(allData);
		return allData;
	}
}
